from transformers import AutoModelForSequenceClassification,AutoTokenizer,pipeline
import torch
#分词器
#加载模型
model = AutoModelForSequenceClassification.from_pretrained("D:\models\multilingual-sentiment-analysis")
#创建分词器
tokenizer = AutoTokenizer.from_pretrained("D:\models\multilingual-sentiment-analysis")
sentence = "今天天气真好"
tokens = tokenizer.tokenize(sentence)
print(tokens)

ids = tokenizer.convert_tokens_to_ids(tokens)
print(ids)

tokens = tokenizer.convert_ids_to_tokens(ids)
print(tokens)

str_sen = tokenizer.convert_tokens_to_string(tokens)
print(str_sen)

ids = tokenizer.encode(sentence)#如果加上add_special_tokens=False，就不会在句首和句尾添加特殊符号
print(ids)

str_sen = tokenizer.decode(ids)
print(str_sen)

ids = tokenizer.encode(sentence,padding="max_length",max_length=15)
print(ids)

ids = tokenizer.encode(sentence,max_length=5,truncation=True)
print(ids)

